
version = '0.2.0'
author  = 'Mike Taylor'
license = """
Copyright (c) 2007 Mike Taylor
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""
tf_bot.py

Scan the Twitter Public Timeline cache (generated by tf_poll.py)
and send any that match to a set of Jabber ID's using xmpppy.

Requires:
    Python          2.5 is what I'm using - haven't tried it on any other
                    http://python.org

    xmpppy          Alexey Nezhdanov
                    http://xmpppy.sourceforge.net/

Contact Info:
    bear            aka Mike Taylor
                    bear42@gmail.com (email and jabber id)
"""

# TODO email target
# TODO and/or compound filters
# TODO lastlog - search N last public tweets for keyword
# TODO allow list to have an optional "brief" mode to allow for backing up and the output
#      should be able to be feed directly to the multi-line option
# TODO have it stop processing on the first match found (global flags to turn this on or off?)
# TODO keep last N message text and remove any that are complete dups (to remove burps/hiccups)

# TODO Find and kill oddball network error from the jabber stack that seems to be happening once a day
# TODO create PID to prevent multiple copies? (not needed right now)
# TODO Load items from command line? (not needed right now)


import sys, os
import codecs
import string
import re
import time
import signal
import select
import traceback
import urllib2
import subprocess

from optparse import OptionParser

import xmpp
import tf_store


_default_template = u'%(screen_name)s: %(post)s'

def dumpException(msg):
    """
    Take current traceback info and make it user-friendly
    """
    t, v, tb = sys.exc_info()

    s  = '+------ error ------+\n'
    s += '%s\n' % time.strftime('%H:%M on %A, %d %B')
    s += '=-------------------=\n'
    s += '%s\n' % msg
    s += '=-------------------=\n'
    s += string.join(traceback.format_exception(t, v, tb), '')
    s += '+-------------------+\n'

    sys.stderr.write(s)
    xT.log(s)


class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
    def http_error_301(self, req, fp, code, msg, headers):
        result = urllib2.HTTPRedirectHandler.http_error_301(self, req, fp, code, msg, headers)              
        result.status = code
        return result

    def http_error_302(self, req, fp, code, msg, headers):
        print '302',code,headers
        result = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)              
        result.status = code
        return result


def getRedirectedUrl(url):
    result = ''
    try:
        request = urllib2.Request(url)
        opener  = urllib2.build_opener()#(SmartRedirectHandler())
        page    = opener.open(request)
        result  = page.url
    except:
        pass

    return result


def scanFilters(JID, text):
    """
    Scans the filter list for the given JID
    Assumes text is passed in lowercased already
    
    Returns
        result      list of filters that matched
        excluded    list of exclude filters that matched
    """
    include = []
    exclude = []

    for (target, exprType, origTemplate, expr, template, regex) in filters[JID]:
        flag  = False
        match = None
        if regex is None:
            if expr in text:
                flag = True
        else:
            match = regex.search(text)
            if match is not None:
                flag = True
        if flag:
            if exprType in ['fx', 'tx', 'rx']:
                exclude.append((target, expr, origTemplate, template, match))
            else:
                include.append((target, expr, origTemplate, template, match))

    return include, exclude


def cmdHelp(fromJID, args):
    helpText = """hello from the Twitter filter bot - Manta.

The purpose of this bot is to allow you to select from the
public twitters and get them sent to you via Jabber.

The following commands are available:

list -- return a numbered list of the filters you currently have defined
from [ex] [to:JID] <twitter username> -- match (or exclude) any post from the given twitter username with optional target JID
text [ex] [to:JID] <text to match> -- add a plain text filter with optional target JID
regex [ex] [to:JID] <regex pattern> -- add a regular expression filter (compiled with IGNORECASE,UNICODE,VERBOSE) with optional target JID
test <text to scan> -- run the given text past the defined filters and retun a result
clear #[, #...] -- clear the given filter # from the filters you have defined
export -- return a message that contains all defined filters
http://<url> -- if given a tinyurl, urltea or tweetl shortened url return the redirected url
last [yyyy-mm-dd] -- return the last set of published results for the current (or given) day

if text or regex is followed by "ex" then the filter becomes an exclude and any items that match are immediately discarded

For example:

text twitter
regex (sxsw|apple|microsoft|scoble)
clear 2
clear 1,3,5
"""
#purge  -- clear all filters you have defined
#stop   -- tell the bot not to send any IM's (they will be queued)
#start  -- tell the bot to send IM's, any queued messages will be sent
#remove -- remove your JID from the system

    return helpText


def getFilters(fromJID):
    JID    = fromJID.getStripped()
    result = []

    if JID in filters:
        result = filters[JID]

    return result

def setFilters(fromJID, newFilters):
    JID      = fromJID.getStripped()
    filename = os.path.join(options.filterDir, JID)
    f        = open(filename, 'w')

    for (target, exprType, origTemplate, expr, template, regex) in newFilters:
        f.write('%s:|:%s:|:%s:|:%s\n' % (target, exprType, origTemplate, expr))

    f.close()
    filters[JID] = newFilters

def listFilters(fromJID):
    filterItems = getFilters(fromJID)
    JID         = fromJID.getStripped()

    if len(filterItems) > 0:
        response = 'Active filters defined for your JID (expr and/or tmpl):\n'
        n = 1
        for (target, exprType, origTemplate, expr, template, regex) in filterItems:
            if exprType in ['r', 'rx']:
                s = '.....regex'
            elif exprType in ['t', 'tx']:
                s = '......text'
            else:
                s = '......from'
            if exprType in ['fx', 'tx', 'rx']:
                s += ' (ex)'
            else:
                s += ''
            response += '# %2d\n%s: %s' % (n, s, expr)
            if len(origTemplate) > 0:
                response += '\n..template: %s' % origTemplate
            if target != JID:
                response += '\n....target: %s' % target
            response += '\n'
            n += 1
    else:
        response = 'Your JID (%s) could not be found in the current filter list' % fromJID.getStripped()

    return response

def cmdList(fromJID, args):
    response = listFilters(fromJID)

    return response

def cmdClear(fromJID, args):
    if len(args) > 0:
        ids   = []
        s     = ' '.join(args).replace(',', ' ')
        items = s.split(' ')

        filterItems = getFilters(fromJID)
        count       = len(filterItems)

        if count > 0:
            for item in items:
                try:
                    filterID = int(item)
                except:
                    filterID = 0
                if filterID > 0 and filterID <= count:
                    if filterID not in ids:
                        ids.append(filterID)

            if len(ids) > 0:
                response   = 'The following filters have been cleared:\n'
                newFilters = []

                n = 1
                for (target, exprType, origTemplate, expr, template, regex) in filterItems:
                    if n in ids:
                        response += 'filter #%2d\nexpr: %s' % (n, expr)
                        if len(origTemplate) > 0:
                            response += '\ntmpl: %s' % origTemplate
                        response += '\n\n'
                    else:
                        newFilters.append([target, exprType, origTemplate, expr, template, regex])

                    n += 1

                setFilters(fromJID, newFilters)

                response += '\n\n%s' % listFilters(fromJID)
            else:
                response = "Your original list of filter id's [%s] generated an empty list" % s
        else:
            response = 'Your current filter list is empty'

        return response

def cmdFrom(fromJID, args):
    print fromJID, args
    if len(args) > 0:
        if args[0] == 'ex' and len(args) > 1:
            exclude = True
            args    = args[1:]
            print 'exclude filter', args
        else:
            exclude = False

        if (len(args) > 1) and args[0].startswith('to:'):
            target = args[0][3:]
            args   = args[1:]
            print 'setting alternate JID target to [%s]' % target
        else:
            target = fromJID.getStripped()
            print 'setting target to [%s]' % target

        s           = ' '.join(args)
        filterItems = getFilters(fromJID)

        if exclude:
            t = 'fx'
        else:
            t = 'f'

        filterItems.append([target, t, u'', s, u'%(screen_name)s %(post)s', None])

        setFilters(fromJID, filterItems)

        response = listFilters(fromJID)

        return response

def cmdText(fromJID, args):
    print fromJID, args
    if len(args) > 0:
        if args[0] == 'ex' and len(args) > 1:
            exclude = True
            args    = args[1:]
            print 'exclude filter', args
        else:
            exclude = False

        if (len(args) > 1) and args[0].startswith('to:'):
            target = args[0][3:]
            args   = args[1:]
            print 'setting alternate JID target to [%s]' % target
        else:
            target = fromJID.getStripped()
            print 'setting target to [%s]' % target

        s           = ' '.join(args)
        filterItems = getFilters(fromJID)

        if exclude:
            t = 'tx'
        else:
            t = 't'

        filterItems.append([target, t, u'', s, u'%(screen_name)s %(post)s', None])

        setFilters(fromJID, filterItems)

        response = listFilters(fromJID)

        return response

def cmdRegex(fromJID, args):
    if len(args) > 0:
        if args[0] == 'ex' and len(args) > 1:
            exclude = True
            args    = args[1:]
            print 'exclude filter', args
        else:
            exclude = False

        if (len(args) > 1) and args[0].startswith('to:'):
            target = args[0][3:]
            args   = args[1:]
            print 'setting alternate JID target to [%s]' % target
        else:
            target = fromJID.getStripped()
            print 'setting target to [%s]' % target

        s           = ' '.join(args)
        filterItems = getFilters(fromJID)
        error       = ''
        try:
            regex = re.compile(s, re.IGNORECASE + re.UNICODE + re.VERBOSE)
        except:
            t, error, tb = sys.exc_info()
            regex = None

        if regex is None:
            response  = 'An error occurred while creating your regex filter item.\n'
            response += 'The regex engine complained about: %s\n' % error
        else:
            if exclude:
                t = 'rx'
            else:
                t = 'r'
            filterItems.append([target, t, u'', s, u'%(screen_name)s %(post)s', regex])

            setFilters(fromJID, filterItems)

            response = listFilters(fromJID)

        return response

def dumpFilterSet(filterList):
    response = ''

    for target, expr, origTemplate, template, match in filterList:
        response += '\nfilter: %s\n' % expr
        if match is not None:
            d = match.groupdict()
            if d is not None:
                response += '...match.groupdict():\n'

                for key in d:
                    response += '...%s: %s\n' % (key, d[key])

    return response

def cmdTest(fromJID, args):
    if len(args) > 0:
        s   = ' '.join(args)
        JID = fromJID.getStripped()

        include, exclude = scanFilters(JID, s)

        if len(include) > 0 or len(exclude) > 0:
            if len(include) > 0:
                response  = 'Your test text matched for inclusion.'
                response += dumpFilterSet(include)
            if len(exclude) > 0:
                response = 'Your test text matched for exclusion.'
                response += dumpFilterSet(exclude)
        else:
            response = 'Your test text did not match.'

        return response

def cmdExport(fromJID, args):
    filterItems = getFilters(fromJID)
    JID         = fromJID.getStripped()

    if len(filterItems) > 0:
        response = ''
        for (target, exprType, origTemplate, expr, template, regex) in filterItems:
            if exprType in ['r', 'rx']:
                response += 'regex '
            elif exprType in ['t', 'tx']:
                response += 'text '
            else:
                response += 'from '
            if exprType in ['fx', 'tx', 'rx']:
                response += 'ex '
            if target != JID:
                response += 'to:%s ' % target
            response += '%s\n' % expr
    else:
        response = 'No filters defined'

    return response

def cmdStats(fromJID, args):
    return buildStatus()

def cmdLast(fromJID, args):
    JID  = fromJID.getStripped()
    date = time.strftime('%Y-%m-%d')

    if len(args) > 0:
        date = args[0]

    items    = sT.findLast(JID, date=date)
    dItem    = {}
    response = u'Posts sent to you on %s (%d)' % (date, len(items))

    for item in items:
        for key in ['post_id', 'post_timestamp', 'screen_name', 'name', 'location',
                    'imageurl', 'url', 'desc', 'post']:
                    dItem[key] = getattr(item, key, '')

        response += u'\n%s' % (_default_template % dItem)

    print len(response)

    return response


commands = { 'help':   cmdHelp,
             'list':   cmdList,
             'clear':  cmdClear,
             'text':   cmdText,
             'regex':  cmdRegex,
             'test':   cmdTest,
             'export': cmdExport,
             'from':   cmdFrom,
             'stats':  cmdStats,
             'last':   cmdLast,
             #'purge':  cmdPurge,
             #'stop':   cmdStop,
             #'start':  cmdStart,
             #'remove': cmdRemove,
           }


class tf_bot:
    """
    """
    def __init__(self, fromJID, fromPW, statusJID, debug):
        """
        fromJID     JID assigned to the bot. I would recommend using
                    a resource description of /bot - anything really
                    as long as it's not empty.  E.g. code-bar@jabber.org/bot
        fromPW      Password required to connect as fromJID
        statusJID   JID to send status information about the bot's activities
        debug       Turns on the low-level xmpppy debug messages (very chatty)
        """
        if debug:
            d = ['always']
        else:
            d = []

        self.active   = False
        self.jid      = xmpp.protocol.JID(fromJID)
        self.password = fromPW
        self.client   = xmpp.Client(self.jid.getDomain(), debug=d)

        self.msgCount = 0
        self.cmdCount = 0

        if statusJID is not None:
            self.statusJID = xmpp.protocol.JID(statusJID)
        else:
            self.statusJID = None

    def start(self):
        """
        Connect to the network and authenticate.
        Register the required handlers and sends the initial presence update.
        """
        self.active = False

        print 'xmpp: connecting'

        if self.client.connect():
            print 'xmpp: authenticating'

            if self.client.auth(self.jid.getNode(), self.password, resource=self.jid.getResource()):
                self.client.RegisterHandler('message', self.on_message)
                self.client.RegisterHandler('presence', self.on_presence)

                # first remove the default disconnect handler (it just raises an IOError)
                # and then register our own super-powerful handler
                self.client.UnregisterDisconnectHandler(self.client.DisconnectHandler)
                self.client.RegisterDisconnectHandler(self.disconnected)

                self.client.sendInitPresence()

                  # yes, the xmpppy lib uses a string to show connection info
                self.active = self.client.isConnected != ''

                self.log('manta: connected')
            else:
                sys.stderr.write('unable to authenticate\n')
        else:
            sys.stderr.write('unable to connect\n')

    def disconnected(self):
        """
        Disconnect handler
        
        Called if the xmpppy protocol layer detects a dropped link
        """
        self.active = False

    def on_message(self, con, event):
        """
        Message handler
        """
        eventType = event.getType()
        fromJID   = event.getFrom()
        JID       = fromJID.getStripped()
        body      = event.getBody()

        self.msgCount += 1

        if body is None:
            print eventType, fromJID, body
        else:
            lines = body.split('\n')
            for line in lines:
                response = None
                items    = line.split(' ')

                if len(items) > 0:
                    cmd = items[0].lower().strip()

                    if cmd in commands:
                        self.log('cmd: %s %s' % (fromJID, cmd))

                        try:
                            response = commands[cmd](fromJID, items[1:])
                        except:
                            dumpException('Exception processing command [%s]' % cmd)
                            response = 'An internal error has occurred while trying to process your last command.'

                        self.cmdCount += 1

                    elif cmd.startswith('http'):
                        if (u'://urltea' in cmd) or (u'://tinyurl' in cmd) or (u'://tweetl' in cmd):
                            response = u'%s --> %s' % (cmd, getRedirectedUrl(cmd))
                            self.cmdCount += 1

                if response is not None:
                    reply = event.buildReply(text=response)
                    reply.setType(event.getType())    # reply to chat w/chat

                    try:
                        xT.client.send(reply)
                    except:
                        dumpException('Exception while trying to send the response for command [%s]' % cmd)

    def on_presence(self, con, event):
        """
        Presence Handler
        
        Currently not used.
        """
        fromJID  = xmpp.protocol.JID(event.getFrom())
        presType = event.getType()

        # auto-respond to subscribes but don't subscribe back
        if presType == 'subscribe':
            self.client.send(xmpp.protocol.Presence(to=event.getFrom(),frm=event.getTo(),typ='subscribed'))

        print 'xmpp: presence: %s %s %s %s %s' % (fromJID, presType, event.getShow(), event.getStatus(), event.getPriority())

    def send(self, to, text):
        """
        Send chat message to a single JID
        
        Helper function that takes a JID string and the message text and
        creates the xmpp objects required to send the message.
        """
        if self.active:
            jid = xmpp.protocol.JID(to)
            msg = xmpp.protocol.Message(to=jid, body=text, typ='chat')

            self.client.send(msg)

    def log(self, text):
        """
        Send log message to twitd
        """
        if self.active:
            if self.statusJID is None:
                sys.stdout.write('%s\n' % text)
            else:
                self.client.send(xmpp.protocol.Message(to=self.statusJID, body=text))


def filterPosts(toJID):
    """
    Load the current set of public timeline posts
    
    loops thru the list of new posts and then send them to a JID.
    """
    global filters, botStats

    skipped   = 0
    polled    = 0
    published = 0
    offline   = 0
    excluded  = 0
    targets   = 0
    fcount    = 0
    queue     = {}

    roster = xT.client.getRoster().getItems()

    for item in sT.getNewPosts():
        polled += 1

        dItem = {}
        for key in ['post_id', 'post_timestamp', 'screen_name', 'name', 'location', 
                    'imageurl', 'url', 'desc', 'post']:
            dItem[key] = getattr(item, key, '')

        #print dItem

        s  = u'%(post_id)s:|:%(post_timestamp)s:|:'
        s += u'%(screen_name)s:|:%(name)s:|:%(location)s:|:'
        s += u'%(imageurl)s:|:%(url)s:|:'
        s += u'%(desc)s:|:%(post)s'

        key         = '%s_%s' % (item.post_timestamp, item.post_id)
        queue[key]  = s % dItem
        loweredText = item.post.lower()

        # search list of JID keyed filters
        for filterJID in filters:
            targets += 1
            if filterJID in roster:
                include, exclude = scanFilters(filterJID, loweredText)

                fcount += (len(include) + len(exclude))

                if len(include) > 0:
                    if len(exclude) > 0:
                        excluded += 1
                    else:
                        sT.postHistoryEntry(filterJID, item)

                        for target, expr, origTemplate, template, match in include:
                            published += 1

                            JID = xmpp.protocol.JID(target)
                            xT.client.send(xmpp.protocol.Message(to=JID, body=template % dItem))
                            print 'pushing to', JID
            else:
                offline += 1

        sT.markFiltered(item.post_id)

    index = queue.keys()
    index.sort()

    for JID in toJID:
        if JID in roster:
            for key in index:
                xT.client.send(xmpp.protocol.Message(to=JID, body=queue[key]))
        else:
            print 'ircJID %s offline' % JID

    botStats['targets']    = targets
    botStats['filters']    = fcount
    botStats['polled']    += polled
    botStats['published'] += published

    print 'public timeline: polled %d, skipped %d, sent %d, offline %d, excluded %d, published %d' % (polled, skipped, len(index), offline, excluded, published)


def loadFilters():
    """
    Load the JID keyed list of filters
    
    For now it reads from a file.
    """
    global filters, botStats

    filters = {}
    tCount  = 0
    eCount  = 0

    for filenameJID in os.listdir(options.filterDir):
        tCount += 1
        f = open(os.path.join(options.filterDir, filenameJID), 'r')
        for line in f:
            data = line[:-1].split(':|:', 5)

            #TARGET:|:EXPR TYPE:|:TEMPLATE:|:EXPR
            if len(data) == 4:
                target = data[0]

                if filenameJID not in filters:
                    filters[filenameJID] = []

                eCount += 1

                if len(data[2]) == 0:
                    template = _default_template
                else:
                    template = data[2]

                regex = None
                if data[1] in ['r', 'rx']:
                    try:
                        regex = re.compile(data[3], re.IGNORECASE + re.UNICODE + re.VERBOSE)
                    except:
                        dumpException('filter regex')
                        regex = None

                data.append(template)
                data.append(regex)

                filters[filenameJID].append(data)

        f.close()

    botStats['users'] = tCount

    xT.log('Loaded %d expressions for %d users' % (eCount, tCount))


def checkConfig():
    """
    Parse command-line for the config filename.
    Load config file and setup options.
    """
    cfg = { 'botJID':    None,
            'botPW':     None,
            'ircJID':    None,
            'statusJID': None,
            'filterDir': './jid_filters',
            'dbPath':    '',
          }

    parser = OptionParser(version=version)

    parser.add_option('-c', '--config',    dest='configFile', default='tf.cfg')
    parser.add_option('-j', '--jid',       dest='botJID',     default=cfg['botJID'])
    parser.add_option('-p', '--password',  dest='botPW',      default=cfg['botPW'])
    parser.add_option('-i', '--ircbot',    dest='ircJID',     default=cfg['ircJID'])
    parser.add_option('-s', '--statusbot', dest='statusJID',  default=cfg['statusJID'])
    parser.add_option('-f', '--filters',   dest='filterDir',  default=cfg['filterDir'])
    parser.add_option('',   '--dbPath',    dest='dbPath',     default=cfg['dbPath'])

    (options, args) = parser.parse_args()
    options.args    = args

    if os.path.isfile(options.configFile):
        for item in open(options.configFile, 'r').readlines():
            if len(item) > 1:
                key, value = item[:-1].split('=', 1)
                key = key.strip()
                if key in cfg.keys():
                    setattr(options, key, value.strip())

    if options.botJID is None:
        print 'Unable to run without a botJID defined'
        sys.exit(1)

    options.filterDir = os.path.abspath(options.filterDir)

    if not os.path.isdir(options.filterDir):
        print '%s directory not found - please create it if missing' % options.filterDir
        sys.exit(1)

    if options.ircJID is None:
        options.ircJID = []
    else:
        options.ircJID = options.ircJID.split(',')

    return options


def buildStatus():
    global botStats

    r = subprocess.Popen(['ls', '-la'], stdout=subprocess.PIPE).communicate()[0]

    botStats['messages'] += xT.msgCount
    botStats['commands'] += xT.cmdCount
    botStats['records']   = sT.reccount()
    botStats['dirinfo']   = ''.join(r)

    sT.postStatsEntry(botStats)

    s = """bot status:
%(records)d records
%(users)d users
%(targets)d targets
%(filters)d filters
%(polled)d polled
%(published)d published
%(messages)d messages
%(commands)d commands
-+-+-+-
%(dirinfo)s""" % (botStats)

    return s


def sendStatus():
    xT.log(buildStatus())


def signalHandler(signum, frame):
    """
    Any signals received just dump out - nothing fancy
    """
    sys.stderr.write('Caught kill signal - terminating\n')
    sys.exit(signum)


if __name__ == "__main__":
    options = checkConfig()

    signal.signal(signal.SIGINT,  signalHandler)
    signal.signal(signal.SIGTERM, signalHandler)
    #signal.signal(signal.SIGHUP, signalHandler)

    global xT, sT, filters, botStats

    sT = tf_store.Store(options.dbPath)

    filters  = {}
    botStats = { 'commands':  0,
                 'messages':  0,
                 'records':   0,
                 'users':     0,
                 'targets':   0,
                 'filters':   0,
                 'polled':    0,
                 'published': 0,
               }

    while True:
        xT = None

        try:
            # JID and password the bot uses for Jabber
            xT = tf_bot(options.botJID, options.botPW, options.statusJID, debug=False)

            xT.start()

            loadFilters()

            lastPingTime = time.time() - 3600
            services     = {}

            services[xT.client.Connection._sock] = 0

            while xT.active:
                try:
                    (i, o, e) = select.select(services.keys(), [], [], 20)

                    for service in i:
                        if services[service] == 0:
                            try:
                                xT.client.Process(1)
                            except:
                                dumpException('exception during xmpp loop')

                    try:
                        filterPosts(options.ircJID)
                    except:
                        dumpException('exception during filterPosts()')

                    if time.time() - lastPingTime > 3600:
                        lastPingTime = time.time()
                        sendStatus()

                except:
                    dumpException('error during select loop')
                    time.sleep(10)
        except:
            dumpException('error during outer loop')
            time.sleep(10)

    sT.close()

